
runProcess <- function() {
  library(XML)
  library(stringr)
  library(RCurl)
  folder <- 'C:/SalesInsight/CompanyProject/Data/'
  indCodeFile <- 'C:/SalesInsight/CompanyProject/HooverCodes.txt'
  
  indInfo <- read.table(file=str_c(indCodeFile), quote='',header=FALSE, sep='|', col.names=c('code','Name','Count'), colClasses=c('numeric','character','numeric') )
  
  for(p in 1:nrow(indInfo)) {
    code <- indInfo$code[p]
    name <- indInfo$Name[p]
    print(str_c('processing for industry - ', name))
    downloadCompanies(code, folder, name)
 }
}

downloadCompanies <- function(code, folder, name) {
  baseLink <- str_c('http://www.hoovers.com/company-information/company-search.html?maxitems=100&nvcnt=76&nvind=',code,'&nvloc=1')
  fileName <- str_c(folder, code,'.txt')  
  logFile <- str_c(folder, 'log/',code, '_errorLinks.txt')
  count <- tryCatch({
    value <- getCount(baseLink)
  }, error = function(e) {
    print(str_c('error getting count for ', name))
    value <- 0
  })

  if(count > 0) {
    for (y in 1:ceiling(count/100)) {
      print(str_c('processing for page - ', y, ' of ', ceiling(count/100)))
      finalLink <- str_c(baseLink,'&page=',y)
      tryCatch({
        table <- readHTMLTable(finalLink)
        compInfo <- cbind(table[[1]])
        write.table(col.names=FALSE, quote=FALSE, compInfo,file=fileName,append=TRUE, sep='|' ,eol='\r\n', row.names=FALSE)
      }, error = function(e) {
         write(finalLink, logFile, ncolumns=1, append=TRUE) 
      })
     }
  } else {
    print(str_c('No records found for industry  ', name))
  }
}

getCount <- function(finalLink) {
  firstPage <- htmlParse(finalLink)
  noOfPages <- getNodeSet(firstPage,'//p[@class=\'range\']')
  countDF <- xmlToDataFrame(noOfPages)
  actualCount <- as.numeric(str_split(countDF$text[1], "of")[[1]][2])
  return (actualCount)
}